package cn.jdy.tools;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import cn.jdy.framework.util.HttpUtil;


/**
 * 从网络抓取资源 第一步：列表抓取(列表网址,网页编码) 第二步：内容抓取
 */
public class FindContent {
  /**
   * 文章内容中需要剔除的代码和标记
   */
  private static final String CONTENT_REX = "(?i)(<iframe([^>]*)>([\\s\\S]*)</iframe>|"
                                            + "<script([^>]*)>([\\s\\S]*)</script>|"
                                            + "<style([^>]*)>([\\s\\S]*)</style>|"
                                            + "<div([^>]*)>|</div>|"
                                            + "<a([^>]*)>|</a>|"
                                            + "<!--([\\s\\S]*)-->)";
    public static void main(String[] args) {
        try {
            // 第一步：获取采集列表内容
            String url = "http://care.39.net/sjbj/";
            String charset = "gb2312";
            String startFlag = "<div class=\"partbox\">";
            String endFlag = "<div class=\"con_right\">";
            String content = HttpUtil.sendByUserAgent(url, charset, 18500);
            getLists(content,startFlag,endFlag);
            
            // 第二步：获取文章内容
//            String url = "http://care.39.net/a/150429/4615346.html";
//            String charset = "gb2312";
//            String startFlag = "<div class=\"art_con\" id=\"contentText\">";
//            String endFlag = "<div class=\"art_page\" id=\"art_page\">";
//            
//            // 模拟浏览器获取
//            String html = HttpUtil.sendByUserAgent(url, charset);
//            System.out.println(html);
//            String content = getAreaContent(html,startFlag,endFlag);
//
//            // 过滤不需要的内容：例如 广告，脚本等
//            System.out.println(content.replaceAll(CONTENT_REX, ""));
            
            // 图片转换
            
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    
    /**
     * 获取采集链接列表
     */
    public static void getLists(String s,String startFlag,String endFlag){
        // 获取标题
        String title = getTitle(s);
        // 去掉指定后缀
//        title = title.replaceAll("_39健康保健_39健康网", "");  
        System.out.println(title);
        
        // 获取区域内容
        String area = getAreaContent(s,startFlag,endFlag);
        
        // 获取超链接
        List<String> links = getLink(area);
        for (String link : links) {
            System.out.println(link);
        }
    }
    
    /**
     * 获取指定区域的内容
     * <div class=\"partbox\">(.*?)<div class=\"con_right\">
     */
    public static String getAreaContent(String s,String startFlag,String endFlag) {
        // 获取指定区域的内容
        final Pattern pa = Pattern.compile(startFlag+"(.*?)"+endFlag, Pattern.DOTALL);  
        final Matcher ma = pa.matcher(s);  
        StringBuffer sb = new StringBuffer(); 
        
        while (ma.find()) {  
            sb.append(ma.group(1));
        }  
        return sb.toString();
    }
    
    /**
     * 获得网页标题
     */
    public static String getTitle(final String s) {
        String regex;
        String title = "";
        final List<String> list = new ArrayList<String>();
        regex = "<title>.*?</title>";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            title = title + list.get(i);
        }
        return outTag(title);
    }
    
    /**
     * @return 获得超链接
     */
    public static List<String> getLink(final String s) {
        String regex;
        final List<String> list = new ArrayList<String>();
        regex = "<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>";
        final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            String href = ma.group(2);
            if(href.matches("(http|https)://.*")){
                list.add(href);
            }
//            System.out.println("(2)==>"+ma.group(2));
//            System.out.println("(5)==>"+ma.group(5));
        }
        return list;
    }
    
    /**
     * @return 去掉标记
     */
    public static String outTag(final String s) {
        return s.replaceAll("<.*?>", "");
    }
    
    /**
     * @return 获得脚本代码
     */
    public static List<String> getScript(final String s) {
        String regex;
        final List<String> list = new ArrayList<String>();
        regex = "<script.*?</script>";
        final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        return list;
    }
    
    /**
     * @return 获得CSS
     */
    public List<String> getCSS(final String s) {
        String regex;
        final List<String> list = new ArrayList<String>();
        regex = "<style.*?</style>";
        final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        return list;
    }
}
